/**
 * Project SearchEngine
 */

#ifndef _PAGELIBPREPROCESSOR_H
#define _PAGELIBPREPROCESSOR_H

#include "WordSegmentation.h"
#include "WebPage.h"
#include "SplitTool.h"
#include <vector>
#include <unordered_map>
#include <utility>
#include <string>


using std::pair;
using std::make_pair;
using std::string;
using std::unordered_map;

class PageLibPreprocessor {
public: 
    PageLibPreprocessor();
    ~PageLibPreprocessor();
    
    void readInfoFromFile();
    void cutRedundantPage();
    
    void bulidInvertIndexMap();
    
    void storeOnDisk();
private: 
    WordSegmentation _jieba;
    vector<WebPage> _pageList;
    unordered_map<int, pair<int,int>> _offsetLib;
    unordered_map<string, vector<pair<int,double>>>  _invertIndexLib;
    /* SplitTool* _wordCutter; */
};

#endif //_PAGELIBPREPROCESSOR_H
